#!/bin/bash

#PBS -o out
#PBS -e err
#PBS -N @[JBNME]
#PBS -A @[ACCNR]
#PBS -q @[QUEUE]
#PBS -l place=vscatter,select=@[NODES]:ncpus=@[NCPUS]:mpiprocs=@[TPN]:ompthreads=@[THRD]:mem=500G
#PBS -l place=excl
#PBS -l walltime=00:@[WLCLK]:00

set -eux

cd $PBS_O_WORKDIR

echo -n " $( date +%s )," >  job_timestamp.txt

set +x
module use $PWD/modulefiles
module load modules.fv3
module load cray-pals
module list
set -x

echo "Model started:  " `date`

export OMP_STACKSIZE=512M
export OMP_NUM_THREADS=@[THRD]
export OMP_PLACES=cores
export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
export ESMF_RUNTIME_PROFILE=ON
export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"

# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run.
if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then
    echo "The job should abort now, with exit status 1." 1>&2
    echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2
    false
fi

mpiexec -n @[TASKS] -ppn @[TPN] --cpu-bind core -depth @[THRD] ./fv3.exe

echo "Model ended:    " `date`
echo -n " $( date +%s )," >> job_timestamp.txt
